package ua.com.stormlabs;

import ua.com.stormlabs.jsitemapper.LinkPool;
import ua.com.stormlabs.jsitemapper.Crawler;
import ua.com.stormlabs.jsitemapper.LinkExtractor;
import ua.com.stormlabs.jsitemapper.LinkPreprocessor;

import java.net.MalformedURLException;
import java.util.List;
import java.util.LinkedList;

/**
 * User: storm
 * Date: 26.03.2009
 */
public class Main {

    public static void main(String[] args) throws MalformedURLException {
        System.out.println("Hello, world!");
        LinkPool linkPool = LinkPool.getInstance();
        String host = "http://auto.qa.smartus.ru";
        String startUrl = "http://auto.qa.smartus.ru/catalog";

        linkPool.addAwaiting(startUrl);
        LinkExtractor linkExtractor = LinkExtractor.getInstance();
        linkExtractor.setHost(host);
        linkExtractor.addExtractorPattern("\\bhref=[\"']([^'^\"]*)[\"']");

        linkExtractor.addFilterPattern(".*[.]css$");
        linkExtractor.addFilterPattern(".*[.]ico$");
        linkExtractor.addFilterPattern("javascript://.*");
        linkExtractor.addFilterPattern(".*/redirector[/]?.*");
        linkExtractor.addFilterPattern(".*/redirect[/]?.*");
        linkExtractor.addFilterPattern(".*/classifieds.*");
        linkExtractor.addFilterPattern(".*/news.*/\\d+");
        linkExtractor.addFilterPattern(".*/sendMessage([?].*)?");

        LinkPreprocessor linkPreprocessor = LinkPreprocessor.getInstance();
        linkPreprocessor.addRetainParamsPattern("");

        int threadCount = 20;

//        String proxyHost = "192.168.0.1";
//        int proxyPort = 808;
        List<Crawler> crawlers = new LinkedList<Crawler>();
        for (int i = 0; i < threadCount; i++) {
            Crawler crawler = new Crawler(i);
//            crawler.setPause(5000L);
//            crawler.setProxyHost(proxyHost);
//            crawler.setProxyPort(proxyPort);
            crawler.go();
            crawlers.add(crawler);
        }
        System.out.println("Waiting for all crawlers");
        for (Crawler crawler : crawlers) {
            crawler.join();
        }
        System.out.println("All crawlers done.");
        linkPool.close();
    }
}
